﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Linq.Expressions;
using MySql.Data.MySqlClient;
using System.Data.Linq;
using System.Net.Http;
using System.Collections.Concurrent;
using MuYue.Common;
using MuYue;
using MuYue.ORM;
using MuYue.Log;
using System.Text.RegularExpressions;
using System.Threading;
using System.IO;

namespace Json.Net.Test {


    /// <summary>
    /// 网络爬虫
    /// </summary>
    public class Crawel : IDisposable {

        /// <summary>
        /// 网络抓取的数据缓存的目录
        /// </summary>
        private readonly string filePath;

        /// <summary>
        /// 最大工作线程数
        /// </summary>
        private readonly int maxThreads;

        /// <summary>
        /// 网页抓取处理之后获取到的网址
        /// </summary>
        private ThreadModel<string> rq;

        /// <summary>
        /// 已经处理的数据的缓存信息
        /// 如果在这个缓存队里面则表示已经处理过了哦!
        /// </summary>
        private HashSet<long> cache;

        /// <summary>
        /// 等待开始抓取的内容队列
        /// </summary>
        private ConcurrentQueue<WebModel> wq;

        /// <summary>
        /// 将处理过的对象添加到缓存数据里面
        /// </summary>
        /// <param name="model"></param>
        private void AddIntoCache(WebModel model) {
            if (!cache.Contains(model.ID)) {
                cache.Add(model.ID);
            }
        }

        private string host = "http://www.yssm.org/";

        /// <summary>
        /// 将结果进行验证存档等等处理
        /// </summary>
        /// <param name="url">获取的新的url地址</param>
        private void DealResult(string url) {
            //Console.WriteLine("rq__working...");
            if (url.StartsWith(host)) {
                var hash = ComputeHash(url);
                var model = GetModelFromDataBase(url);
                if (model == null) {
                    model = new WebModel(url);
                    model.Hash = hash;
                    AddIntoWorkingQuene(model);
                    LogService.Debug(url);
                }
            }
            //1.校验url是否符合规则信息
            //2.计算hash
            //3.数据库中检查任务
            //4.存在就抛弃结果
            //5.如果不存在存档并且发送到工作队列等待处理
        }

        public Crawel() {

            //获取配置信息
            var configMgr = new XmlConfigManager("./config/app.conf");
            this.filePath = configMgr.TryGetValue("Crawel", "FilePath", "./www");
            this.maxThreads = Int32.Parse(configMgr.TryGetValue("Crawel", "MaxThreads", "20"));

            cache = new HashSet<long>();
            wq = new ConcurrentQueue<WebModel>();
            rq = new ThreadModel<string>();
            rq.DealModelEvent += DealResult;
            if (!Directory.Exists(filePath)) {
                Directory.CreateDirectory(filePath);
            }


            iq.DealModelEvent += DownloadImage;
        }

        private void DownloadImage(string url) {
            try {
                var fileName = imagefolder + ComputeHash(url) + ".jpg";
                LogService.Debug(fileName);
                if (!File.Exists(fileName))
                    Mrsoft.Common.WebEngine.DownloadFile(url, fileName);
            }
            catch { }
        }

        private ThreadModel<string> iq = new ThreadModel<string>();

        private string imagefolder;

        public void Start(string url, int threadNum) {
            var uri = new Uri(url);
            host = "http://" + new Uri(url).Host;
            imagefolder = "./image/" + uri.Host.Replace("/", "") + "/";
            if (!Directory.Exists(imagefolder)) {
                Directory.CreateDirectory(imagefolder);
            }


            var model = GetModelFromDataBase(url);
            if (model == null) {
                model = new WebModel(url);
                model.Hash = ComputeHash(url);
                AddIntoWorkingQuene(model);
            }
            //parentId = cache.First();
            //创建线程开始处理
            StartWorker(0);
            Thread.Sleep(5000);
            for (int i = 1; i < threadNum; i++)
                StartWorker(i);
        }

        HttpClient client = new HttpClient();

        Regex match = new Regex("<a.*?href=[\"'](.*?)[\"'].*?>(.*?)</a>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
        Regex imageMatch = new Regex("<img.*?src=[\"'](.*?)[\"'].*?>", RegexOptions.IgnoreCase | RegexOptions.Compiled);

        private void Worker(object obj) {
            LogService.Debug("{1} working...{0}...", wq.Count, obj);
            WebModel m;
            if (wq.TryDequeue(out m)) {
                try {
                    var html = Mrsoft.Common.WebEngine.DownloadString(m.Url);
                    if (!string.IsNullOrEmpty(html)) {

                        var fileName = Path.Combine(filePath, m.Hash + ".html");
                        File.WriteAllText(fileName, html, Encoding.UTF8);
                        //正则匹配所有a标签
                        var rs = match.Matches(html);
                        //LogService.Debug("匹配数{0}", rs.Count);
                        foreach (Match r in rs) {

                            var url = r.Result("$1");
                            var str = r.Result("$2");
                            if (url.StartsWith("#"))
                                continue;
                            url = ComplieUrl(m.Url, url);
                            rq.Append(url);

                        }
                        m.State = GrabState.Done;
                        m.LastUpdate = DateTime.Now;
                        FastDb.Update(m, "id");

                        //抓取所有的Image
                        //
                        var im = imageMatch.Matches(html);

                        foreach (Match tm in im) {

                            var url = tm.Result("$1");
                            if (url.StartsWith("#"))
                                continue;

                            iq.Append(ComplieUrl(m.Url, url));

                        }
                        Thread.Sleep(500);
                    }
                    StartWorker(obj);
                }
                catch (Exception ex) {
                    LogService.Warn(ex.Message);
                    StartWorker(obj);
                }
            }
        }

        private string ComplieUrl(string now, string next) {
            if (!next.StartsWith("http")) {
                int index = 0;
                if (next.StartsWith("/")) {
                    index = now.IndexOf("/", 10);
                }
                else {
                    index = now.LastIndexOf("/");
                }
                next = now.Substring(0, index) + next;
            }
            return next;
        }

        private void StartWorker(object obj) {
            System.Threading.ThreadPool.QueueUserWorkItem(Worker, obj);
        }





        /// <summary>
        /// 将需要处理的网页添加到处理进程里面
        /// </summary>
        /// <param name="url"></param>
        private void AddIntoWorkingQuene(WebModel model) {
            FastDb.Add(model);
            var m = GetModelFromDataBase(model.Url);
            if (m != null) {
                wq.Enqueue(m);
            }
            else {
                MuYue.Log.LogService.Warn("网站[{0}]的数据读取失败", model.Url);
            }
        }

        /// <summary>
        /// 冲数据库读取
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        private WebModel GetModelFromDataBase(string url) {
            var hash = ComputeHash(url);
            var where = string.Format("`hash` ='{0}'", hash);
            var m = FastDb.Get<WebModel>(where);
            return m;
        }

        private string ComputeHash(string url) {
            return Utility.Md5Encoding(url);
        }


        #region IDisposable 成员

        public void Dispose() {
            if (rq != null) {
                rq.Dispose();
                rq = null;
            }
        }

        #endregion
    }

}


